package com.chance.cc.crawler.development.bootstrap.dcdapp;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRecordFilter.*;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Site_Info;

/**
 * 懂车帝 口碑、问答启动类
 */
public class DCDAppStart {

    private static final String domain = "dcdapp";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
//        proxy.setUsername("HEW657EL99F83S9D");
        proxy.setUsername("H7307T4706B25G4D");
//        proxy.setPassword("8916B1F3F10B1979");
        proxy.setPassword("05B4877CC39192C0");
    }
    public static void main(String[] args) {

        //车系采集
       seriesCrawlerRecord();
     // seriesCrawlerRecordTest();
        //itemCrawlerRecordTest();

    }

    private static void seriesCrawlerRecord(){
        String url = "https://www.dcdapp.com/";
        String site = "praise";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPage)
                .domain(domain)
                .httpUrl(url)
                .httpConfig(HttpConfig.me(domain))
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24  * 105,null))
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
//                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
//                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);

        CrawlerRecord commentFilter = new CrawlerRequestRecord();
        commentFilter.setFilter(keyOrDateRange);
        commentFilter.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-","filter",domain,site,"queue")));
        commentFilter.addFilterInfo(FilterUtils.dateRangeFilterInfo(24*105,null));
        crawlerRequestRecord.tagsCreator().resultTags().getCategoryTag().addKVTag("comment_filter_info", JSON.toJSONString(commentFilter));
//        "praise","forum"
        List<String> crawlerModule = Arrays.asList("praise","forum");
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("crawlerModule",crawlerModule);

        CrawlerRequestRecord keywordRecord = CrawlerRequestRecord.builder()
                .startPageRequest("dcdapp_series_keyword",turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/dcdapp/keys?site=everyDayRealtime")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
                .crawlerThreadNum(10)
                .consoleResultPipeline("kafka")//控制台输出
                //.consoleResultPipeline("redis")//控制台输出
                .fileResultPipeline("redis","F:\\chance_log\\dcdapp\\redis_tmp.log",false) //文件输出
                .requestRecord(crawlerRequestRecord)
                .supportRecord(keywordRecord)
                .build("com.chance.cc.crawler.development.scripts.dcdapp")
                .start();
    }

    private static void seriesCrawlerRecordTest(){
        String url = "https://www.dongchedi.com/auto/series/214";
        String site = "praise";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPage)
                .domain(domain)
                .httpUrl(url)
                .httpConfig(HttpConfig.me(domain))
                .filter(dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*100,null))
//                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);
        crawlerRequestRecord.tagsCreator().bizTags().addSiteBiz(site);
        crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().addKVTag("domainName","dongchedi");

        CrawlerRecord commentFilter = new CrawlerRequestRecord();
        commentFilter.setFilter(key);
        commentFilter.addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null));
        commentFilter.addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain));
        crawlerRequestRecord.tagsCreator().resultTags().getCategoryTag().addKVTag("comment_filter_info", JSON.toJSONString(commentFilter));
//        "praise","forum"
//        List<String> crawlerModule  = Arrays.asList("praise","forum");
//        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("crawlerModule",crawlerModule);


        DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
                .crawlerThreadNum(5)
                .consoleResultPipeline("kafka")//控制台输出
                .fileResultPipeline("kafka","F:\\chance_log\\dcdapp\\kafka.log",false) //文件输出
                .fileResultPipeline("redis","F:\\chance_log\\dcdapp\\redis-simple.log",false) //文件输出
                .requestRecord(crawlerRequestRecord)
                .build("com.chance.cc.crawler.development.scripts.dcdapp")
                .start();
    }

    private static void itemCrawlerRecordTest(){
        String url = "https://www.dongchedi.com/koubei/1679043722279936";//https://www.dongchedi.com/ugc/article/1706686936615936
        String site = "praise";
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, turnPageItem)
                .domain(domain)
                .httpUrl(url)
                .httpConfig(HttpConfig.me(domain))
                .filter(dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 7,null))
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .releaseTime(1621096465000L)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(domain);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(site);
        crawlerRequestRecord.tagsCreator().bizTags().addSiteBiz(site);
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(Tag_Site_Info,"口碑");//site_info //板块信息说明 按板块采集添加此biz标签区分数据

        CrawlerRecord commentFilter = new CrawlerRequestRecord();
        commentFilter.setFilter(key);
        commentFilter.addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1606060800000L,1608566400000L}));
        commentFilter.addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain));
        crawlerRequestRecord.tagsCreator().resultTags().getCategoryTag().addKVTag("comment_filter_info", JSON.toJSONString(commentFilter));
//        "praise","forum"
        List<String> crawlerModule  = Arrays.asList("praise","forum");
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("crawlerModule",crawlerModule);


        DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))
                .consoleResultPipeline("kafka")//控制台输出
//                .consoleResultPipeline("redis")//控制台输出
                .fileResultPipeline("kafka","F:\\chance_log\\dcdapp\\kafka-simple.log",false) //文件输出
//                .fileResultPipeline("redis","F:\\chance_log\\dcdapp\\redis.log",false) //文件输出
                .requestRecord(crawlerRequestRecord)
                .build("com.chance.cc.crawler.development.scripts.dcdapp")
                .start();
    }

    @Test
    public void test1(){
        long l = 1627960506764l;
        long l1 = 1612193041000l;
        //1627968259000
        //1581730149000
        //1627809610000
        //1612193041000
        long p = 1627461641487L;
        Date date1 = new Date();
        Date date = new Date(l1);
        SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

        System.out.println(sd.format(date));
        /*try {
            //System.out.println(DateUtils.parseDate("2021-07-28 17:22", "yyyy-MM-dd HH:mm").getTime());
        } catch (ParseException e) {
            e.printStackTrace();
        }*/
        try {
            String encode = URLEncoder.encode("{\"page_id\":\"page_forum_home\",\"product_name\":\"pc\"}", "UTF-8");
            //System.out.println(encode);
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
    }
    @Test
    public void test2(){
        String url = "\"https://www.dongchedi.com/motor/pc/car/series/get_review_list?\" +\n" +
                "            \"series_id=%s&count=15&page=%s&sort_by=create_time&only_owner=0&part_id=S0&tab=\"";
        String.format(url,"1154","1");
        url = url + "%E7%BB%BC%E5%90%88";
        System.out.println(url);
    }
}
